package com.example;
import org.openqa.selenium.*;
import org.openqa.selenium.NoSuchElementException;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import io.github.bonigarcia.wdm.WebDriverManager;
import java.io.*;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;

public class DouyinOptimizedCrawler {
    private static final long MIN_DELAY = 3000;  // 最小请求间隔（毫秒）
    private static final long MAX_DELAY = 8000;  // 最大请求间隔（毫秒）
    private final String keyword;
    private final int maxVideos;
    private final Set<String> processedUrls = new HashSet<>();
    private final List<String> videoData = new ArrayList<>();
    private WebDriver driver;
    private final AtomicInteger totalCount = new AtomicInteger(0);
    private final AtomicInteger errorCount = new AtomicInteger(0);

    public DouyinOptimizedCrawler(String keyword, int maxVideos) {
        this.keyword = keyword;
        this.maxVideos = maxVideos;
        loadProcessedUrls();
        System.out.println("初始化完成，准备爬取关于 '" + keyword + "' 的 " + maxVideos + " 条视频...");
    }

    public void run() {
        try {
            // 初始化WebDriver
            initDriver();

            // 登录
            login();

            // 搜索关键词
            search();

            // 开始爬取
            crawl();

            // 导出数据
            exportData();

            System.out.println("===== 爬取完成 =====");
            System.out.println("成功: " + totalCount.get() + " 条");
            System.out.println("失败: " + errorCount.get() + " 条");

        } catch (Exception e) {
            System.err.println("爬取过程中发生致命错误: " + e.getMessage());
            e.printStackTrace();
        } finally {
            if (driver != null) {
                driver.quit();
            }
            saveProcessedUrls();
        }
    }

    private void initDriver() {
        WebDriverManager.chromedriver().setup();

        ChromeOptions options = new ChromeOptions();
        options.addArguments("--disable-blink-features=AutomationControlled");
        options.addArguments("--user-agent=" + getRandomUserAgent());
        options.addArguments("--window-size=1920,1080");
        options.addArguments("--disable-notifications");
        options.addArguments("--disable-popup-blocking");
        options.addArguments("--start-maximized");

        // 禁用WebDriver特征
        options.setExperimentalOption("excludeSwitches", Collections.singletonList("enable-automation"));
        options.setExperimentalOption("useAutomationExtension", false);

        driver = new ChromeDriver(options);
        driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);
        System.out.println("WebDriver初始化完成");
    }

    private String getRandomUserAgent() {
        String[] agents = {
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0",
                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"
        };
        return agents[new Random().nextInt(agents.length)];
    }

    private void login() throws InterruptedException {
        driver.get("https://www.douyin.com/");
        System.out.println("请在打开的浏览器中手动扫码登录抖音，登录后按回车继续...");

        Scanner scanner = new Scanner(System.in);
        scanner.nextLine();

        // 验证登录状态
        Thread.sleep(3000);
        try {
            WebElement userAvatar = waitForElement(
                    By.xpath("//img[contains(@class, 'avatar')] | //div[contains(@class, 'user-avatar')]"),
                    5
            );
            System.out.println("检测到已登录用户");
        } catch (Exception e) {
            System.err.println("登录验证失败，可能未成功登录");
            System.err.println("继续尝试爬取，但可能会受到限制");
        }

        randomDelay();
    }

    private void search() throws InterruptedException {
        driver.get("https://www.douyin.com/search/" + keyword);
        System.out.println("搜索关键词 '" + keyword + "'");

        // 等待页面加载
        randomDelay();

        // 尝试切换到视频标签页（新版抖音可能结构不同）
        try {
            WebElement videoTab = waitForElement(
                    By.xpath("//div[contains(text(), '视频') and contains(@class, 'tab-item')] | //a[contains(text(), '视频')]"),
                    8
            );
            videoTab.click();
            System.out.println("已切换到视频标签页");
        } catch (Exception e) {
            System.err.println("未找到视频标签页，可能页面结构已更新");
            System.err.println("将继续尝试从当前页面提取视频");
        }

        randomDelay();
    }

    private void crawl() throws InterruptedException {
        int page = 0;

        while (totalCount.get() < maxVideos) {
            page++;
            System.out.println("正在处理第 " + page + " 页...");

            // 随机延迟
            randomDelay();

            // 滚动页面，加载更多内容
            for (int i = 0; i < 3; i++) {
                scrollPage();
                randomDelay();
            }

            // 提取视频链接（新版抖音视频卡片结构）
            List<WebElement> videoCards = driver.findElements(By.xpath(
                    "//div[contains(@class, 'video-card')] | //div[contains(@class, 'aweme-item')] | //a[contains(@href, '/video/')]"
            ));

            System.out.println("本页找到 " + videoCards.size() + " 个视频卡片");

            for (WebElement card : videoCards) {
                if (totalCount.get() >= maxVideos) {
                    System.out.println("已达到最大爬取数量，停止爬取");
                    return;
                }

                try {
                    // 从卡片中提取视频链接
                    WebElement linkElement = card.findElement(By.cssSelector("a[href*='/video/']"));
                    String videoUrl = linkElement.getAttribute("href");

                    if (videoUrl == null || !videoUrl.startsWith("https://www.douyin.com/video/")) {
                        continue;
                    }

                    // 标准化URL（去除参数）
                    videoUrl = videoUrl.split("\\?")[0];

                    // 检查是否已处理
                    if (processedUrls.contains(videoUrl)) {
                        System.out.println("跳过已处理视频: " + videoUrl);
                        continue;
                    }

                    // 处理视频
                    if (processVideo(videoUrl)) {
                        // 标记为已处理
                        processedUrls.add(videoUrl);

                        // 增加计数
                        int count = totalCount.incrementAndGet();
                        if (count % 10 == 0) {
                            System.out.println("进度: " + count + "/" + maxVideos);
                        }
                    }

                } catch (Exception e) {
                    System.err.println("处理视频卡片时出错: " + e.getMessage());
                    errorCount.incrementAndGet();
                }
            }

            // 模拟翻页行为
            randomDelay();
        }
    }

    private void scrollPage() {
        try {
            int scrollDistance = 500 + new Random().nextInt(500);
            ((JavascriptExecutor) driver).executeScript("window.scrollBy(0, " + scrollDistance + ");");
            System.out.println("页面向下滚动 " + scrollDistance + " 像素");

            // 等待页面加载
            Thread.sleep(1000 + new Random().nextInt(1000));
        } catch (Exception e) {
            System.err.println("滚动页面失败: " + e.getMessage());
        }
    }

    private boolean processVideo(String videoUrl) {
        randomDelay(); // 请求前随机延迟

        try {
            // 打开视频页面
            driver.get(videoUrl);
            System.out.println("正在处理视频: " + videoUrl);

            // 等待页面加载（增加等待时间）
            randomDelay();

            // 检查是否被反爬拦截
            checkBlocked();

            // 提取数据
            String description = escapeCsv(extractDescription());
            String duration = escapeCsv(extractDuration());
            long likes = extractLikes();
            long comments = extractComments();
            String musicTitle = escapeCsv(extractMusicTitle());
            long favorites = extractFavorites();

            // 验证数据有效性
            if (description.isEmpty() && musicTitle.isEmpty()) {
                System.err.println("数据提取失败，可能被反爬拦截: " + videoUrl);
                return false;
            }

            // 保存数据
            String dataLine = String.format("%s,%s,%d,%d,%s,%d,%s",
                    description, duration, likes, comments, musicTitle, favorites, videoUrl);

            videoData.add(dataLine);
            System.out.println("成功爬取视频: " + description.substring(0, Math.min(30, description.length())) + "...");

            return true;

        } catch (Exception e) {
            System.err.println("处理视频 " + videoUrl + " 失败: " + e.getMessage());
            errorCount.incrementAndGet();

            // 检测是否被封禁
            if (isBlocked()) {
                System.err.println("检测到IP被封禁，程序将暂停30分钟...");
                try {
                    Thread.sleep(30 * 60 * 1000); // 暂停30分钟
                } catch (InterruptedException ie) {
                    Thread.currentThread().interrupt();
                }
            }

            return false;
        }
    }

    private void checkBlocked() {
        try {
            // 检查是否出现验证码页面
            WebElement captchaElement = driver.findElement(By.xpath(
                    "//div[contains(text(), '验证码')] | //div[contains(text(), '安全验证')]"
            ));
            System.err.println("检测到验证码页面，请在浏览器中手动处理");

            // 等待用户处理验证码
            Scanner scanner = new Scanner(System.in);
            System.out.println("处理完验证码后按回车继续...");
            scanner.nextLine();
        } catch (Exception e) {
            // 没有找到验证码元素，继续执行
        }
    }

    private boolean isBlocked() {
        try {
            // 检查是否被封禁（根据实际情况调整XPath）
            WebElement blockedElement = driver.findElement(By.xpath(
                    "//div[contains(text(), '访问受限')] | //div[contains(text(), '异常请求')]"
            ));
            return true;
        } catch (Exception e) {
            return false;
        }
    }

    private String escapeCsv(String value) {
        if (value == null) return "";
        boolean needsQuotes = value.contains(",") || value.contains("\"") || value.contains("\n");
        if (needsQuotes) {
            return "\"" + value.replace("\"", "\"\"") + "\"";
        }
        return value;
    }

    private WebElement waitForElement(By locator, int timeoutSeconds) {
        return new WebDriverWait(driver, Duration.ofSeconds(timeoutSeconds))
                .until(ExpectedConditions.visibilityOfElementLocated(locator));
    }

    private String extractDescription() {
        try {
            // 多种定位策略，提高兼容性
            List<By> strategies = Arrays.asList(
                    By.xpath("//h1[contains(@class, 'video-title')]"),
                    By.xpath("//span[contains(@class, 'desc')]"),
                    By.xpath("//div[contains(@class, 'text-descr')]"),
                    By.xpath("//div[contains(@class, 'video-desc')]")
            );

            return findElementWithStrategies(strategies, 10).getText().trim();
        } catch (Exception e) {
            System.err.println("提取描述失败: " + e.getMessage());
            return "未知描述";
        }
    }

    private String extractDuration() {
        try {
            List<By> strategies = Arrays.asList(
                    By.xpath("//span[contains(@class, 'time')]"),
                    By.xpath("//div[contains(@class, 'duration')]"),
                    By.xpath("//div[contains(@class, 'video-time')]")
            );

            return findElementWithStrategies(strategies, 8).getText().trim();
        } catch (Exception e) {
            System.err.println("提取时长失败: " + e.getMessage());
            return "未知时长";
        }
    }

    private long extractLikes() {
        try {
            List<By> strategies = Arrays.asList(
                    By.xpath("//span[contains(@class, 'like-count')]"),
                    By.xpath("//button[contains(@class, 'like-btn')]//span"),
                    By.xpath("//div[contains(@class, 'stats-item')]//span[contains(text(), '万')]")
            );

            return parseCount(findElementWithStrategies(strategies, 8).getText().trim());
        } catch (Exception e) {
            System.err.println("提取点赞数失败: " + e.getMessage());
            return 0;
        }
    }

    private long extractComments() {
        try {
            List<By> strategies = Arrays.asList(
                    By.xpath("//span[contains(@class, 'comment-count')]"),
                    By.xpath("//button[contains(@class, 'comment-btn')]//span"),
                    By.xpath("//div[contains(@class, 'stats-item') and contains(., '评论')]//span")
            );

            return parseCount(findElementWithStrategies(strategies, 8).getText().trim());
        } catch (Exception e) {
            System.err.println("提取评论数失败: " + e.getMessage());
            return 0;
        }
    }

    private String extractMusicTitle() {
        try {
            List<By> strategies = Arrays.asList(
                    By.xpath("//a[contains(@href, '/music/')]//span"),
                    By.xpath("//div[contains(@class, 'music-title')]"),
                    By.xpath("//div[contains(text(), '原声')]/following-sibling::div")
            );

            return findElementWithStrategies(strategies, 8).getText().trim();
        } catch (Exception e) {
            System.err.println("提取音乐标题失败: " + e.getMessage());
            return "未知音乐";
        }
    }

    private long extractFavorites() {
        try {
            List<By> strategies = Arrays.asList(
                    By.xpath("//span[contains(@class, 'collect-count')]"),
                    By.xpath("//button[contains(@class, 'collect-btn')]//span"),
                    By.xpath("//div[contains(@class, 'stats-item') and contains(., '收藏')]//span")
            );

            return parseCount(findElementWithStrategies(strategies, 8).getText().trim());
        } catch (Exception e) {
            System.err.println("提取收藏数失败: " + e.getMessage());
            return 0;
        }
    }

    private WebElement findElementWithStrategies(List<By> strategies, int timeoutSeconds) {
        for (By strategy : strategies) {
            try {
                return waitForElement(strategy, timeoutSeconds);
            } catch (Exception e) {
                // 尝试下一个策略
            }
        }
        throw new NoSuchElementException("所有定位策略都失败");
    }

    private long parseCount(String countText) {
        try {
            if (countText.isEmpty()) return 0;
            if (countText.contains("万")) {
                return (long) (Double.parseDouble(countText.replaceAll("[^0-9.]", "")) * 10000);
            } else if (countText.contains("亿")) {
                return (long) (Double.parseDouble(countText.replaceAll("[^0-9.]", "")) * 100000000);
            } else {
                return Long.parseLong(countText.replaceAll("[^0-9]", ""));
            }
        } catch (Exception e) {
            System.err.println("解析计数失败: " + countText + ", 错误: " + e.getMessage());
            return 0;
        }
    }

    private void randomDelay() {
        long delay = MIN_DELAY + new Random().nextLong(MAX_DELAY - MIN_DELAY);
        try {
            System.out.println("等待 " + (delay/1000) + " 秒...");
            Thread.sleep(delay);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
    }

    private void exportData() {
        try (BufferedWriter writer = new BufferedWriter(new FileWriter("douyin_data_" + keyword + ".csv"))) {
            writer.write("description,duration,likes,comments,music_title,favorites,url\n");

            for (String data : videoData) {
                writer.write(data + "\n");
            }

            System.out.println("数据已导出至: douyin_data_" + keyword + ".csv");

        } catch (IOException e) {
            System.err.println("导出数据失败: " + e.getMessage());
        }
    }

    private void loadProcessedUrls() {
        File file = new File("processed_urls_" + keyword + ".txt");
        if (file.exists()) {
            try (BufferedReader reader = new BufferedReader(new FileReader(file))) {
                String line;
                while ((line = reader.readLine()) != null) {
                    processedUrls.add(line.trim());
                }
                System.out.println("已加载 " + processedUrls.size() + " 个已处理URL");
            } catch (IOException e) {
                System.err.println("加载已处理URL失败: " + e.getMessage());
            }
        }
    }

    private void saveProcessedUrls() {
        try (BufferedWriter writer = new BufferedWriter(new FileWriter("processed_urls_" + keyword + ".txt"))) {
            for (String url : processedUrls) {
                writer.write(url + "\n");
            }
            System.out.println("已保存 " + processedUrls.size() + " 个已处理URL");
        } catch (IOException e) {
            System.err.println("保存已处理URL失败: " + e.getMessage());
        }
    }

    public static void main(String[] args) {
        // 爬取500条关于"美食"的视频
        DouyinOptimizedCrawler crawler = new DouyinOptimizedCrawler("美食", 500);
        crawler.run();
    }
}